package com.wanglei.rdd.dependency

import org.apache.spark.rdd.RDD
import org.apache.spark.storage.StorageLevel
import org.apache.spark.{SparkConf, SparkContext}

object Spark03_rddcache {

  def main(args: Array[String]): Unit = {
    // application
    val sparkConf = new SparkConf().setMaster("local[2]").setAppName("wc")
    val sc = new SparkContext(sparkConf)

    //
    val lines: RDD[String] = sc.textFile("datas/1.txt")
    val words: RDD[String] = lines.flatMap(_.split(" "))
    val wordToOne: RDD[(String, Int)] = words.map(word => (word, 1))

    // cache or persist
    wordToOne.cache()
    wordToOne.persist(StorageLevel.MEMORY_AND_DISK)

    val res1: RDD[(String, Int)] = wordToOne.reduceByKey(_ + _)
    val res2: RDD[(String, Iterable[Int])] = wordToOne.groupByKey()


    // job1
    res1.collect()
    // job2
    res2.collect()
    //
    sc.stop()
  }

}
